{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyalink.alink import *\n",
    "useLocalEnv(1)\n",
    "\n",
    "from utils import *\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "DATA_DIR = ROOT_DIR + \"tmall\" + os.sep\n",
    "\n",
    "ORIGIN_FILE = \"action_log.csv\";\n",
    "\n",
    "FEATURE_LABEL_FILE = \"feature_label.ak\";\n",
    "TRAIN_FILE = \"train.ak\";\n",
    "TEST_FILE = \"test.ak\";\n",
    "TRAIN_SAMPLE_FILE = \"train_sample.ak\";\n",
    "\n",
    "LABEL_COL_NAME = \"label\";\n",
    "PREDICTION_COL_NAME = \"pred\";\n",
    "PRED_DETAIL_COL_NAME = \"predInfo\";\n",
    "\n",
    "def getSource(): \n",
    "    return CsvSourceBatchOp()\\\n",
    "            .setFilePath(DATA_DIR + ORIGIN_FILE)\\\n",
    "            .setSchemaStr(\"user_id long, brand_id long, type int, ts timestamp\")\\\n",
    "            .setFieldDelimiter(\"\\t\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_1\n",
    "source = getSource();\n",
    "\n",
    "source.lazyPrint(10, \"origin file\");\n",
    "\n",
    "source.lazyPrintStatistics(\"stat of origin file\");\n",
    "\n",
    "source.link(\n",
    "    CorrelationBatchOp()\\\n",
    "        .setSelectedCols([\"user_id\", \"brand_id\", \"type\"])\\\n",
    "        .lazyPrintCorrelation()\n",
    ");\n",
    "\n",
    "source.select(\"min(ts) AS min_ts, max(ts) AS max_ts\").lazyPrint(-1);\n",
    "\n",
    "source.groupBy(\"type\", \"type, COUNT(*) AS cnt\").lazyPrint(-1);\n",
    "\n",
    "BatchOperator.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_3\n",
    "source = getSource();\n",
    "\n",
    "t1 = source.filter(\"ts < CAST('2014-07-16 00:00:00' AS TIMESTAMP)\");\n",
    "t2 = source.filter(\"ts >= CAST('2014-07-16 00:00:00' AS TIMESTAMP)\");\n",
    "\n",
    "t1.lazyPrint(10, \"t1\").lazyPrintStatistics(\"t1\");\n",
    "\n",
    "t2.lazyPrint(10, \"t2\").lazyPrintStatistics(\"t2\");\n",
    "\n",
    "BatchOperator.execute()\n",
    "\n",
    "clausePreProc = \"user_id, brand_id, type, ts, past_days,\"\\\n",
    "+ \"case when type=0 then 1 else 0 end AS is_click,\"\\\n",
    "+ \"case when type=1 then 1 else 0 end AS is_buy,\"\\\n",
    "+ \"case when type=2 then 1 else 0 end AS is_collect,\"\\\n",
    "+ \"case when type=3 then 1 else 0 end AS is_cart,\"\\\n",
    "+ \"case when type=0 and past_days<=30 then 1 else 0 end AS is_click_1m,\"\\\n",
    "+ \"case when type=1 and past_days<=30 then 1 else 0 end AS is_buy_1m,\"\\\n",
    "+ \"case when type=2 and past_days<=30 then 1 else 0 end AS is_collect_1m,\"\\\n",
    "+ \"case when type=3 and past_days<=30 then 1 else 0 end AS is_cart_1m,\"\\\n",
    "+ \"case when type=0 and past_days<=60 then 1 else 0 end AS is_click_2m,\"\\\n",
    "+ \"case when type=1 and past_days<=60 then 1 else 0 end AS is_buy_2m,\"\\\n",
    "+ \"case when type=2 and past_days<=60 then 1 else 0 end AS is_collect_2m,\"\\\n",
    "+ \"case when type=3 and past_days<=60 then 1 else 0 end AS is_cart_2m,\"\\\n",
    "+ \"case when type=0 and past_days<=90 then 1 else 0 end AS is_click_3m,\"\\\n",
    "+ \"case when type=1 and past_days<=90 then 1 else 0 end AS is_buy_3m,\"\\\n",
    "+ \"case when type=2 and past_days<=90 then 1 else 0 end AS is_collect_3m,\"\\\n",
    "+ \"case when type=3 and past_days<=90 then 1 else 0 end AS is_cart_3m,\"\\\n",
    "+ \"case when type=0 and past_days>30 and past_days<=60 then 1 else 0 end AS is_click_m2nd,\"\\\n",
    "+ \"case when type=1 and past_days>30 and past_days<=60 then 1 else 0 end AS is_buy_m2nd,\"\\\n",
    "+ \"case when type=2 and past_days>30 and past_days<=60 then 1 else 0 end AS is_collect_m2nd,\"\\\n",
    "+ \"case when type=3 and past_days>30 and past_days<=60 then 1 else 0 end AS is_cart_m2nd,\"\\\n",
    "+ \"case when type=0 and past_days>60 and past_days<=90 then 1 else 0 end AS is_click_m3th,\"\\\n",
    "+ \"case when type=1 and past_days>60 and past_days<=90 then 1 else 0 end AS is_buy_m3th,\"\\\n",
    "+ \"case when type=2 and past_days>60 and past_days<=90 then 1 else 0 end AS is_collect_m3th,\"\\\n",
    "+ \"case when type=3 and past_days>60 and past_days<=90 then 1 else 0 end AS is_cart_m3th,\"\\\n",
    "+ \"case when type=0 and past_days<=3 then 1 else 0 end AS is_click_3d,\"\\\n",
    "+ \"case when type=1 and past_days<=3 then 1 else 0 end AS is_buy_3d,\"\\\n",
    "+ \"case when type=2 and past_days<=3 then 1 else 0 end AS is_collect_3d,\"\\\n",
    "+ \"case when type=3 and past_days<=3 then 1 else 0 end AS is_cart_3d,\"\\\n",
    "+ \"case when type=0 and past_days>3 and past_days<=6 then 1 else 0 end AS is_click_3d2nd,\"\\\n",
    "+ \"case when type=1 and past_days>3 and past_days<=6 then 1 else 0 end AS is_buy_3d2nd,\"\\\n",
    "+ \"case when type=2 and past_days>3 and past_days<=6 then 1 else 0 end AS is_collect_3d2nd,\"\\\n",
    "+ \"case when type=3 and past_days>3 and past_days<=6 then 1 else 0 end AS is_cart_3d2nd,\"\\\n",
    "+ \"case when type=0 and past_days>6 and past_days<=9 then 1 else 0 end AS is_click_3d3th,\"\\\n",
    "+ \"case when type=1 and past_days>6 and past_days<=9 then 1 else 0 end AS is_buy_3d3th,\"\\\n",
    "+ \"case when type=2 and past_days>6 and past_days<=9 then 1 else 0 end AS is_collect_3d3th,\"\\\n",
    "+ \"case when type=3 and past_days>6 and past_days<=9 then 1 else 0 end AS is_cart_3d3th,\"\\\n",
    "+ \"case when type=0 and past_days<=7 then 1 else 0 end AS is_click_1w,\"\\\n",
    "+ \"case when type=1 and past_days<=7 then 1 else 0 end AS is_buy_1w,\"\\\n",
    "+ \"case when type=2 and past_days<=7 then 1 else 0 end AS is_collect_1w,\"\\\n",
    "+ \"case when type=3 and past_days<=7 then 1 else 0 end AS is_cart_1w,\"\\\n",
    "+ \"case when type=0 and past_days>7 and past_days<=14 then 1 else 0 end AS is_click_w2nd,\"\\\n",
    "+ \"case when type=1 and past_days>7 and past_days<=14 then 1 else 0 end AS is_buy_w2nd,\"\\\n",
    "+ \"case when type=2 and past_days>7 and past_days<=14 then 1 else 0 end AS is_collect_w2nd,\"\\\n",
    "+ \"case when type=3 and past_days>7 and past_days<=14 then 1 else 0 end AS is_cart_w2nd,\"\\\n",
    "+ \"case when type=0 and past_days>14 and past_days<=21 then 1 else 0 end AS is_click_w3th,\"\\\n",
    "+ \"case when type=1 and past_days>14 and past_days<=21 then 1 else 0 end AS is_buy_w3th,\"\\\n",
    "+ \"case when type=2 and past_days>14 and past_days<=21 then 1 else 0 end AS is_collect_w3th,\"\\\n",
    "+ \"case when type=3 and past_days>14 and past_days<=21 then 1 else 0 end AS is_cart_w3th\";\n",
    "\n",
    "t1_preproc = t1\\\n",
    "    .select(\"user_id, brand_id, type, ts, \"\n",
    "            + \"TIMESTAMPDIFF(DAY, ts, TIMESTAMP '2014-07-16 00:00:00') AS past_days\")\\\n",
    "    .select(clausePreProc);\n",
    "\n",
    "clauseUserBrand = \"user_id, brand_id, SUM(is_click) as cnt_click, SUM(is_buy) as cnt_buy, \"\\\n",
    "+ \"SUM(is_collect) as cnt_collect, SUM(is_cart) as cnt_cart, \"\\\n",
    "+ \"SUM(is_click_1m) as cnt_click_1m, SUM(is_buy_1m) as cnt_buy_1m, \"\\\n",
    "+ \"SUM(is_collect_1m) as cnt_collect_1m, SUM(is_cart_1m) as cnt_cart_1m, \"\\\n",
    "+ \"SUM(is_click_2m) as cnt_click_2m, SUM(is_buy_2m) as cnt_buy_2m, \"\\\n",
    "+ \"SUM(is_collect_2m) as cnt_collect_2m, SUM(is_cart_2m) as cnt_cart_2m, \"\\\n",
    "+ \"SUM(is_click_3m) as cnt_click_3m, SUM(is_buy_3m) as cnt_buy_3m, \"\\\n",
    "+ \"SUM(is_collect_3m) as cnt_collect_3m, SUM(is_cart_3m) as cnt_cart_3m, \"\\\n",
    "+ \"SUM(is_click_m2nd) as cnt_click_m2nd, SUM(is_buy_m2nd) as cnt_buy_m2nd, \"\\\n",
    "+ \"SUM(is_collect_m2nd) as cnt_collect_m2nd, SUM(is_cart_m2nd) as cnt_cart_m2nd, \"\\\n",
    "+ \"SUM(is_click_m3th) as cnt_click_m3th, SUM(is_buy_m3th) as cnt_buy_m3th, \"\\\n",
    "+ \"SUM(is_collect_m3th) as cnt_collect_m3th, SUM(is_cart_m3th) as cnt_cart_m3th, \"\\\n",
    "+ \"SUM(is_click_3d) as cnt_click_3d, SUM(is_buy_3d) as cnt_buy_3d, \"\\\n",
    "+ \"SUM(is_collect_3d) as cnt_collect_3d, SUM(is_cart_3d) as cnt_cart_3d, \"\\\n",
    "+ \"SUM(is_click_3d2nd) as cnt_click_3d2nd, SUM(is_buy_3d2nd) as cnt_buy_3d2nd, \"\\\n",
    "+ \"SUM(is_collect_3d2nd) as cnt_collect_3d2nd, SUM(is_cart_3d2nd) as cnt_cart_3d2nd, \"\\\n",
    "+ \"SUM(is_click_3d3th) as cnt_click_3d3th, SUM(is_buy_3d3th) as cnt_buy_3d3th, \"\\\n",
    "+ \"SUM(is_collect_3d3th) as cnt_collect_3d3th, SUM(is_cart_3d3th) as cnt_cart_3d3th, \"\\\n",
    "+ \"SUM(is_click_1w) as cnt_click_1w, SUM(is_buy_1w) as cnt_buy_1w, \"\\\n",
    "+ \"SUM(is_collect_1w) as cnt_collect_1w, SUM(is_cart_1w) as cnt_cart_1w, \"\\\n",
    "+ \"SUM(is_click_w2nd) as cnt_click_w2nd, SUM(is_buy_w2nd) as cnt_buy_w2nd, \"\\\n",
    "+ \"SUM(is_collect_w2nd) as cnt_collect_w2nd, SUM(is_cart_w2nd) as cnt_cart_w2nd, \"\\\n",
    "+ \"SUM(is_click_w3th) as cnt_click_w3th, SUM(is_buy_w3th) as cnt_buy_w3th, \"\\\n",
    "+ \"SUM(is_collect_w3th) as cnt_collect_w3th, SUM(is_cart_w3th) as cnt_cart_w3th\";\n",
    "\n",
    "t1_userbrand = t1_preproc.groupBy(\"user_id, brand_id\", clauseUserBrand);\n",
    "\n",
    "clauseUserBrand_Rate = \"user_id,brand_id,\"\\\n",
    "+ \"cnt_click,cnt_buy,cnt_collect,cnt_cart,\"\\\n",
    "+ \"cnt_click_1m,cnt_buy_1m,cnt_collect_1m,cnt_cart_1m,\"\\\n",
    "+ \"cnt_click_2m,cnt_buy_2m,cnt_collect_2m,cnt_cart_2m,\"\\\n",
    "+ \"cnt_click_3m,cnt_buy_3m,cnt_collect_3m,cnt_cart_3m,\"\\\n",
    "+ \"cnt_click_m2nd,cnt_buy_m2nd,cnt_collect_m2nd,cnt_cart_m2nd,\"\\\n",
    "+ \"cnt_click_m3th,cnt_buy_m3th,cnt_collect_m3th,cnt_cart_m3th,\"\\\n",
    "+ \"cnt_click_3d,cnt_buy_3d,cnt_collect_3d,cnt_cart_3d,\"\\\n",
    "+ \"cnt_click_3d2nd,cnt_buy_3d2nd,cnt_collect_3d2nd,cnt_cart_3d2nd,\"\\\n",
    "+ \"cnt_click_3d3th,cnt_buy_3d3th,cnt_collect_3d3th,cnt_cart_3d3th,\"\\\n",
    "+ \"cnt_click_1w,cnt_buy_1w,cnt_collect_1w,cnt_cart_1w,\"\\\n",
    "+ \"cnt_click_w2nd,cnt_buy_w2nd,cnt_collect_w2nd,cnt_cart_w2nd,\"\\\n",
    "+ \"cnt_click_w3th,cnt_buy_w3th,cnt_collect_w3th,cnt_cart_w3th,\"\\\n",
    "+ \"case when cnt_buy>cnt_click then 1.0 when cnt_buy=0 then 0.0 else cnt_buy*1.0/cnt_click end AS \"\\\n",
    "+ \"rt_click2buy,\"\\\n",
    "+ \"case when cnt_buy>cnt_collect then 1.0 when cnt_buy=0 then 0.0 else cnt_buy*1.0/cnt_collect end AS \"\\\n",
    "+ \"rt_collect2buy,\"\\\n",
    "+ \"case when cnt_buy>cnt_cart then 1.0 when cnt_buy=0 then 0.0 else cnt_buy*1.0/cnt_cart end AS \"\\\n",
    "+ \"rt_cart2buy,\"\\\n",
    "+ \"case when cnt_buy_3d>cnt_click_3d then 1.0 when cnt_buy_3d=0 then 0.0 else cnt_buy_3d*1.0/cnt_click_3d\"\\\n",
    "+ \" end AS rt_click2buy_3d,\"\\\n",
    "+ \"case when cnt_buy_3d>cnt_collect_3d then 1.0 when cnt_buy_3d=0 then 0.0 else \"\\\n",
    "+ \"cnt_buy_3d*1.0/cnt_collect_3d end AS rt_collect2buy_3d,\"\\\n",
    "+ \"case when cnt_buy_3d>cnt_cart_3d then 1.0 when cnt_buy_3d=0 then 0.0 else cnt_buy_3d*1.0/cnt_cart_3d \"\\\n",
    "+ \"end AS rt_cart2buy_3d,\"\\\n",
    "+ \"case when cnt_buy_1w>cnt_click_1w then 1.0 when cnt_buy_1w=0 then 0.0 else cnt_buy_1w*1.0/cnt_click_1w\"\\\n",
    "+ \" end AS rt_click2buy_1w,\"\\\n",
    "+ \"case when cnt_buy_1w>cnt_collect_1w then 1.0 when cnt_buy_1w=0 then 0.0 else \"\\\n",
    "+ \"cnt_buy_1w*1.0/cnt_collect_1w end AS rt_collect2buy_1w,\"\\\n",
    "+ \"case when cnt_buy_1w>cnt_cart_1w then 1.0 when cnt_buy_1w=0 then 0.0 else cnt_buy_1w*1.0/cnt_cart_1w \"\\\n",
    "+ \"end AS rt_cart2buy_1w,\"\\\n",
    "+ \"case when cnt_buy_1m>cnt_click_1m then 1.0 when cnt_buy_1m=0 then 0.0 else cnt_buy_1m*1.0/cnt_click_1m\"\\\n",
    "+ \" end AS rt_click2buy_1m,\"\\\n",
    "+ \"case when cnt_buy_1m>cnt_collect_1m then 1.0 when cnt_buy_1m=0 then 0.0 else \"\\\n",
    "+ \"cnt_buy_1m*1.0/cnt_collect_1m end AS rt_collect2buy_1m,\"\\\n",
    "+ \"case when cnt_buy_1m>cnt_cart_1m then 1.0 when cnt_buy_1m=0 then 0.0 else cnt_buy_1m*1.0/cnt_cart_1m \"\\\n",
    "+ \"end AS rt_cart2buy_1m,\"\\\n",
    "+ \"case when cnt_click_3d=0 then 0.0 when cnt_click_3d>=120.0*cnt_click_3d2nd then 120.0 else \"\\\n",
    "+ \"cnt_click_3d*1.0/cnt_click_3d2nd end AS rt_click_3d,\"\\\n",
    "+ \"case when cnt_buy_3d=0 then 0.0 when cnt_buy_3d>=10.0*cnt_buy_3d2nd then 10.0 else \"\\\n",
    "+ \"cnt_buy_3d*1.0/cnt_buy_3d2nd end AS rt_buy_3d,\"\\\n",
    "+ \"case when cnt_collect_3d=0 then 0.0 when cnt_collect_3d>=10.0*cnt_collect_3d2nd then 10.0 else \"\\\n",
    "+ \"cnt_collect_3d*1.0/cnt_collect_3d2nd end AS rt_collect_3d,\"\\\n",
    "+ \"case when cnt_cart_3d=0 then 0.0 when cnt_cart_3d>=20.0*cnt_cart_3d2nd then 20.0 else \"\\\n",
    "+ \"cnt_cart_3d*1.0/cnt_cart_3d2nd end AS rt_cart_3d,\"\\\n",
    "+ \"case when cnt_click_1w=0 then 0.0 when cnt_click_1w>=300.0*cnt_click_w2nd then 300.0 else \"\\\n",
    "+ \"cnt_click_1w*1.0/cnt_click_w2nd end AS rt_click_1w,\"\\\n",
    "+ \"case when cnt_buy_1w=0 then 0.0 when cnt_buy_1w>=15.0*cnt_buy_w2nd then 15.0 else \"\\\n",
    "+ \"cnt_buy_1w*1.0/cnt_buy_w2nd end AS rt_buy_1w,\"\\\n",
    "+ \"case when cnt_collect_1w=0 then 0.0 when cnt_collect_1w>=15.0*cnt_collect_w2nd then 15.0 else \"\\\n",
    "+ \"cnt_collect_1w*1.0/cnt_collect_w2nd end AS rt_collect_1w,\"\\\n",
    "+ \"case when cnt_cart_1w=0 then 0.0 when cnt_cart_1w>=40.0*cnt_cart_w2nd then 40.0 else \"\\\n",
    "+ \"cnt_cart_1w*1.0/cnt_cart_w2nd end AS rt_cart_1w,\"\\\n",
    "+ \"case when cnt_click_1m=0 then 0.0 when cnt_click_1m>=500.0*cnt_click_m2nd then 500.0 else \"\\\n",
    "+ \"cnt_click_1m*1.0/cnt_click_m2nd end AS rt_click_1m,\"\\\n",
    "+ \"case when cnt_buy_1m=0 then 0.0 when cnt_buy_1m>=30.0*cnt_buy_m2nd then 30.0 else \"\\\n",
    "+ \"cnt_buy_1m*1.0/cnt_buy_m2nd end AS rt_buy_1m,\"\\\n",
    "+ \"case when cnt_collect_1m=0 then 0.0 when cnt_collect_1m>=30.0*cnt_collect_m2nd then 30.0 else \"\\\n",
    "+ \"cnt_collect_1m*1.0/cnt_collect_m2nd end AS rt_collect_1m,\"\\\n",
    "+ \"case when cnt_cart_1m=0 then 0.0 when cnt_cart_1m>=50.0*cnt_cart_m2nd then 50.0 else \"\\\n",
    "+ \"cnt_cart_1m*1.0/cnt_cart_m2nd end AS rt_cart_1m\";\n",
    "\n",
    "t1_userbrand = t1_userbrand.select(clauseUserBrand_Rate);\n",
    "\n",
    "clauseUser = \"user_id, \"\\\n",
    "+ \"SUM(is_click) as user_cnt_click, SUM(is_buy) as user_cnt_buy, \"\\\n",
    "+ \"SUM(is_collect) as user_cnt_collect, SUM(is_cart) as user_cnt_cart, \"\\\n",
    "+ \"SUM(is_click_1m) as user_cnt_click_1m, SUM(is_buy_1m) as user_cnt_buy_1m, \"\\\n",
    "+ \"SUM(is_collect_1m) as user_cnt_collect_1m, SUM(is_cart_1m) as user_cnt_cart_1m, \"\\\n",
    "+ \"SUM(is_click_2m) as user_cnt_click_2m, SUM(is_buy_2m) as user_cnt_buy_2m, \"\\\n",
    "+ \"SUM(is_collect_2m) as user_cnt_collect_2m, SUM(is_cart_2m) as user_cnt_cart_2m, \"\\\n",
    "+ \"SUM(is_click_3m) as user_cnt_click_3m, SUM(is_buy_3m) as user_cnt_buy_3m, \"\\\n",
    "+ \"SUM(is_collect_3m) as user_cnt_collect_3m, SUM(is_cart_3m) as user_cnt_cart_3m, \"\\\n",
    "+ \"SUM(is_click_m2nd) as user_cnt_click_m2nd, SUM(is_buy_m2nd) as user_cnt_buy_m2nd, \"\\\n",
    "+ \"SUM(is_collect_m2nd) as user_cnt_collect_m2nd, SUM(is_cart_m2nd) as user_cnt_cart_m2nd, \"\\\n",
    "+ \"SUM(is_click_m3th) as user_cnt_click_m3th, SUM(is_buy_m3th) as user_cnt_buy_m3th, \"\\\n",
    "+ \"SUM(is_collect_m3th) as user_cnt_collect_m3th, SUM(is_cart_m3th) as user_cnt_cart_m3th, \"\\\n",
    "+ \"SUM(is_click_3d) as user_cnt_click_3d, SUM(is_buy_3d) as user_cnt_buy_3d, \"\\\n",
    "+ \"SUM(is_collect_3d) as user_cnt_collect_3d, SUM(is_cart_3d) as user_cnt_cart_3d, \"\\\n",
    "+ \"SUM(is_click_3d2nd) as user_cnt_click_3d2nd, SUM(is_buy_3d2nd) as user_cnt_buy_3d2nd, \"\\\n",
    "+ \"SUM(is_collect_3d2nd) as user_cnt_collect_3d2nd, SUM(is_cart_3d2nd) as user_cnt_cart_3d2nd, \"\\\n",
    "+ \"SUM(is_click_3d3th) as user_cnt_click_3d3th, SUM(is_buy_3d3th) as user_cnt_buy_3d3th, \"\\\n",
    "+ \"SUM(is_collect_3d3th) as user_cnt_collect_3d3th, SUM(is_cart_3d3th) as user_cnt_cart_3d3th, \"\\\n",
    "+ \"SUM(is_click_1w) as user_cnt_click_1w, SUM(is_buy_1w) as user_cnt_buy_1w, \"\\\n",
    "+ \"SUM(is_collect_1w) as user_cnt_collect_1w, SUM(is_cart_1w) as user_cnt_cart_1w, \"\\\n",
    "+ \"SUM(is_click_w2nd) as user_cnt_click_w2nd, SUM(is_buy_w2nd) as user_cnt_buy_w2nd, \"\\\n",
    "+ \"SUM(is_collect_w2nd) as user_cnt_collect_w2nd, SUM(is_cart_w2nd) as user_cnt_cart_w2nd, \"\\\n",
    "+ \"SUM(is_click_w3th) as user_cnt_click_w3th, SUM(is_buy_w3th) as user_cnt_buy_w3th, \"\\\n",
    "+ \"SUM(is_collect_w3th) as user_cnt_collect_w3th, SUM(is_cart_w3th) as user_cnt_cart_w3th\";\n",
    "\n",
    "t1_user = t1_preproc.groupBy(\"user_id\", clauseUser);\n",
    "\n",
    "clauseUser_Rate = \"user_id AS user_id4join,\"\\\n",
    "+ \"user_cnt_click,user_cnt_buy,user_cnt_collect,user_cnt_cart,\"\\\n",
    "+ \"user_cnt_click_1m,user_cnt_buy_1m,user_cnt_collect_1m,user_cnt_cart_1m,\"\\\n",
    "+ \"user_cnt_click_2m,user_cnt_buy_2m,user_cnt_collect_2m,user_cnt_cart_2m,\"\\\n",
    "+ \"user_cnt_click_3m,user_cnt_buy_3m,user_cnt_collect_3m,user_cnt_cart_3m,\"\\\n",
    "+ \"user_cnt_click_m2nd,user_cnt_buy_m2nd,user_cnt_collect_m2nd,user_cnt_cart_m2nd,\"\\\n",
    "+ \"user_cnt_click_m3th,user_cnt_buy_m3th,user_cnt_collect_m3th,user_cnt_cart_m3th,\"\\\n",
    "+ \"user_cnt_click_3d,user_cnt_buy_3d,user_cnt_collect_3d,user_cnt_cart_3d,\"\\\n",
    "+ \"user_cnt_click_3d2nd,user_cnt_buy_3d2nd,user_cnt_collect_3d2nd,user_cnt_cart_3d2nd,\"\\\n",
    "+ \"user_cnt_click_3d3th,user_cnt_buy_3d3th,user_cnt_collect_3d3th,user_cnt_cart_3d3th,\"\\\n",
    "+ \"user_cnt_click_1w,user_cnt_buy_1w,user_cnt_collect_1w,user_cnt_cart_1w,\"\\\n",
    "+ \"user_cnt_click_w2nd,user_cnt_buy_w2nd,user_cnt_collect_w2nd,user_cnt_cart_w2nd,\"\\\n",
    "+ \"user_cnt_click_w3th,user_cnt_buy_w3th,user_cnt_collect_w3th,user_cnt_cart_w3th,\"\\\n",
    "+ \"case when user_cnt_buy>user_cnt_click then 1.0 when user_cnt_buy=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy*1.0/user_cnt_click end AS user_rt_click2buy,\"\\\n",
    "+ \"case when user_cnt_buy>user_cnt_collect then 1.0 when user_cnt_buy=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy*1.0/user_cnt_collect end AS user_rt_collect2buy,\"\\\n",
    "+ \"case when user_cnt_buy>user_cnt_cart then 1.0 when user_cnt_buy=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy*1.0/user_cnt_cart end AS user_rt_cart2buy,\"\\\n",
    "+ \"case when user_cnt_buy_3d>user_cnt_click_3d then 1.0 when user_cnt_buy_3d=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy_3d*1.0/user_cnt_click_3d end AS user_rt_click2buy_3d,\"\\\n",
    "+ \"case when user_cnt_buy_3d>user_cnt_collect_3d then 1.0 when user_cnt_buy_3d=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy_3d*1.0/user_cnt_collect_3d end AS user_rt_collect2buy_3d,\"\\\n",
    "+ \"case when user_cnt_buy_3d>user_cnt_cart_3d then 1.0 when user_cnt_buy_3d=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy_3d*1.0/user_cnt_cart_3d end AS user_rt_cart2buy_3d,\"\\\n",
    "+ \"case when user_cnt_buy_1w>user_cnt_click_1w then 1.0 when user_cnt_buy_1w=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy_1w*1.0/user_cnt_click_1w end AS user_rt_click2buy_1w,\"\\\n",
    "+ \"case when user_cnt_buy_1w>user_cnt_collect_1w then 1.0 when user_cnt_buy_1w=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy_1w*1.0/user_cnt_collect_1w end AS user_rt_collect2buy_1w,\"\\\n",
    "+ \"case when user_cnt_buy_1w>user_cnt_cart_1w then 1.0 when user_cnt_buy_1w=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy_1w*1.0/user_cnt_cart_1w end AS user_rt_cart2buy_1w,\"\\\n",
    "+ \"case when user_cnt_buy_1m>user_cnt_click_1m then 1.0 when user_cnt_buy_1m=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy_1m*1.0/user_cnt_click_1m end AS user_rt_click2buy_1m,\"\\\n",
    "+ \"case when user_cnt_buy_1m>user_cnt_collect_1m then 1.0 when user_cnt_buy_1m=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy_1m*1.0/user_cnt_collect_1m end AS user_rt_collect2buy_1m,\"\\\n",
    "+ \"case when user_cnt_buy_1m>user_cnt_cart_1m then 1.0 when user_cnt_buy_1m=0 then 0.0 else \"\\\n",
    "+ \"user_cnt_buy_1m*1.0/user_cnt_cart_1m end AS user_rt_cart2buy_1m\";\n",
    "\n",
    "t1_user = t1_user.select(clauseUser_Rate);\n",
    "\n",
    "clauseBrand = \"brand_id, \"\\\n",
    "+ \"SUM(is_click) as brand_cnt_click, SUM(is_buy) as brand_cnt_buy, \"\\\n",
    "+ \"SUM(is_collect) as brand_cnt_collect, SUM(is_cart) as brand_cnt_cart, \"\\\n",
    "+ \"SUM(is_click_1m) as brand_cnt_click_1m, SUM(is_buy_1m) as brand_cnt_buy_1m, \"\\\n",
    "+ \"SUM(is_collect_1m) as brand_cnt_collect_1m, SUM(is_cart_1m) as brand_cnt_cart_1m, \"\\\n",
    "+ \"SUM(is_click_2m) as brand_cnt_click_2m, SUM(is_buy_2m) as brand_cnt_buy_2m, \"\\\n",
    "+ \"SUM(is_collect_2m) as brand_cnt_collect_2m, SUM(is_cart_2m) as brand_cnt_cart_2m, \"\\\n",
    "+ \"SUM(is_click_3m) as brand_cnt_click_3m, SUM(is_buy_3m) as brand_cnt_buy_3m, \"\\\n",
    "+ \"SUM(is_collect_3m) as brand_cnt_collect_3m, SUM(is_cart_3m) as brand_cnt_cart_3m, \"\\\n",
    "+ \"SUM(is_click_m2nd) as brand_cnt_click_m2nd, SUM(is_buy_m2nd) as brand_cnt_buy_m2nd, \"\\\n",
    "+ \"SUM(is_collect_m2nd) as brand_cnt_collect_m2nd, SUM(is_cart_m2nd) as brand_cnt_cart_m2nd, \"\\\n",
    "+ \"SUM(is_click_m3th) as brand_cnt_click_m3th, SUM(is_buy_m3th) as brand_cnt_buy_m3th, \"\\\n",
    "+ \"SUM(is_collect_m3th) as brand_cnt_collect_m3th, SUM(is_cart_m3th) as brand_cnt_cart_m3th, \"\\\n",
    "+ \"SUM(is_click_3d) as brand_cnt_click_3d, SUM(is_buy_3d) as brand_cnt_buy_3d, \"\\\n",
    "+ \"SUM(is_collect_3d) as brand_cnt_collect_3d, SUM(is_cart_3d) as brand_cnt_cart_3d, \"\\\n",
    "+ \"SUM(is_click_3d2nd) as brand_cnt_click_3d2nd, SUM(is_buy_3d2nd) as brand_cnt_buy_3d2nd, \"\\\n",
    "+ \"SUM(is_collect_3d2nd) as brand_cnt_collect_3d2nd, SUM(is_cart_3d2nd) as brand_cnt_cart_3d2nd, \"\\\n",
    "+ \"SUM(is_click_3d3th) as brand_cnt_click_3d3th, SUM(is_buy_3d3th) as brand_cnt_buy_3d3th, \"\\\n",
    "+ \"SUM(is_collect_3d3th) as brand_cnt_collect_3d3th, SUM(is_cart_3d3th) as brand_cnt_cart_3d3th, \"\\\n",
    "+ \"SUM(is_click_1w) as brand_cnt_click_1w, SUM(is_buy_1w) as brand_cnt_buy_1w, \"\\\n",
    "+ \"SUM(is_collect_1w) as brand_cnt_collect_1w, SUM(is_cart_1w) as brand_cnt_cart_1w, \"\\\n",
    "+ \"SUM(is_click_w2nd) as brand_cnt_click_w2nd, SUM(is_buy_w2nd) as brand_cnt_buy_w2nd, \"\\\n",
    "+ \"SUM(is_collect_w2nd) as brand_cnt_collect_w2nd, SUM(is_cart_w2nd) as brand_cnt_cart_w2nd, \"\\\n",
    "+ \"SUM(is_click_w3th) as brand_cnt_click_w3th, SUM(is_buy_w3th) as brand_cnt_buy_w3th, \"\\\n",
    "+ \"SUM(is_collect_w3th) as brand_cnt_collect_w3th, SUM(is_cart_w3th) as brand_cnt_cart_w3th\";\n",
    "\n",
    "t1_brand = t1_preproc.groupBy(\"brand_id\", clauseBrand);\n",
    "\n",
    "clauseBrand_Rate = \"brand_id AS brand_id4join,\"\\\n",
    "+ \"brand_cnt_click,brand_cnt_buy,brand_cnt_collect,brand_cnt_cart,\"\\\n",
    "+ \"brand_cnt_click_1m,brand_cnt_buy_1m,brand_cnt_collect_1m,brand_cnt_cart_1m,\"\\\n",
    "+ \"brand_cnt_click_2m,brand_cnt_buy_2m,brand_cnt_collect_2m,brand_cnt_cart_2m,\"\\\n",
    "+ \"brand_cnt_click_3m,brand_cnt_buy_3m,brand_cnt_collect_3m,brand_cnt_cart_3m,\"\\\n",
    "+ \"brand_cnt_click_m2nd,brand_cnt_buy_m2nd,brand_cnt_collect_m2nd,brand_cnt_cart_m2nd,\"\\\n",
    "+ \"brand_cnt_click_m3th,brand_cnt_buy_m3th,brand_cnt_collect_m3th,brand_cnt_cart_m3th,\"\\\n",
    "+ \"brand_cnt_click_3d,brand_cnt_buy_3d,brand_cnt_collect_3d,brand_cnt_cart_3d,\"\\\n",
    "+ \"brand_cnt_click_3d2nd,brand_cnt_buy_3d2nd,brand_cnt_collect_3d2nd,brand_cnt_cart_3d2nd,\"\\\n",
    "+ \"brand_cnt_click_3d3th,brand_cnt_buy_3d3th,brand_cnt_collect_3d3th,brand_cnt_cart_3d3th,\"\\\n",
    "+ \"brand_cnt_click_1w,brand_cnt_buy_1w,brand_cnt_collect_1w,brand_cnt_cart_1w,\"\\\n",
    "+ \"brand_cnt_click_w2nd,brand_cnt_buy_w2nd,brand_cnt_collect_w2nd,brand_cnt_cart_w2nd,\"\\\n",
    "+ \"brand_cnt_click_w3th,brand_cnt_buy_w3th,brand_cnt_collect_w3th,brand_cnt_cart_w3th,\"\\\n",
    "+ \"case when brand_cnt_buy>brand_cnt_click then 1.0 when brand_cnt_buy=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy*1.0/brand_cnt_click end AS brand_rt_click2buy,\"\\\n",
    "+ \"case when brand_cnt_buy>brand_cnt_collect then 1.0 when brand_cnt_buy=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy*1.0/brand_cnt_collect end AS brand_rt_collect2buy,\"\\\n",
    "+ \"case when brand_cnt_buy>brand_cnt_cart then 1.0 when brand_cnt_buy=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy*1.0/brand_cnt_cart end AS brand_rt_cart2buy,\"\\\n",
    "+ \"case when brand_cnt_buy_3d>brand_cnt_click_3d then 1.0 when brand_cnt_buy_3d=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy_3d*1.0/brand_cnt_click_3d end AS brand_rt_click2buy_3d,\"\\\n",
    "+ \"case when brand_cnt_buy_3d>brand_cnt_collect_3d then 1.0 when brand_cnt_buy_3d=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy_3d*1.0/brand_cnt_collect_3d end AS brand_rt_collect2buy_3d,\"\\\n",
    "+ \"case when brand_cnt_buy_3d>brand_cnt_cart_3d then 1.0 when brand_cnt_buy_3d=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy_3d*1.0/brand_cnt_cart_3d end AS brand_rt_cart2buy_3d,\"\\\n",
    "+ \"case when brand_cnt_buy_1w>brand_cnt_click_1w then 1.0 when brand_cnt_buy_1w=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy_1w*1.0/brand_cnt_click_1w end AS brand_rt_click2buy_1w,\"\\\n",
    "+ \"case when brand_cnt_buy_1w>brand_cnt_collect_1w then 1.0 when brand_cnt_buy_1w=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy_1w*1.0/brand_cnt_collect_1w end AS brand_rt_collect2buy_1w,\"\\\n",
    "+ \"case when brand_cnt_buy_1w>brand_cnt_cart_1w then 1.0 when brand_cnt_buy_1w=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy_1w*1.0/brand_cnt_cart_1w end AS brand_cart2buy_1w,\"\\\n",
    "+ \"case when brand_cnt_buy_1m>brand_cnt_click_1m then 1.0 when brand_cnt_buy_1m=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy_1m*1.0/brand_cnt_click_1m end AS brand_rt_click2buy_1m,\"\\\n",
    "+ \"case when brand_cnt_buy_1m>brand_cnt_collect_1m then 1.0 when brand_cnt_buy_1m=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy_1m*1.0/brand_cnt_collect_1m end AS brand_rt_collect2buy_1m,\"\\\n",
    "+ \"case when brand_cnt_buy_1m>brand_cnt_cart_1m then 1.0 when brand_cnt_buy_1m=0 then 0.0 else \"\\\n",
    "+ \"brand_cnt_buy_1m*1.0/brand_cnt_cart_1m end AS brand_rt_cart2buy_1m\";\n",
    "\n",
    "t1_brand = t1_brand.select(clauseBrand_Rate);\n",
    "\n",
    "t1_join = JoinBatchOp()\\\n",
    "    .setSelectClause(\"*\")\\\n",
    "    .setJoinPredicate(\"user_id=user_id4join\")\\\n",
    "    .linkFrom(t1_userbrand, t1_user);\n",
    "\n",
    "t1_join = JoinBatchOp()\\\n",
    "    .setSelectClause(\"*\")\\\n",
    "    .setJoinPredicate(\"brand_id=brand_id4join\")\\\n",
    "    .linkFrom(t1_join, t1_brand);\n",
    "\n",
    "t2_label = t2\\\n",
    "    .filter(\"type=1\")\\\n",
    "    .select(\"user_id AS user_id4label, brand_id AS brand_id4label, 1 as label\")\\\n",
    "    .distinct();\n",
    "\n",
    "feature_label = LeftOuterJoinBatchOp()\\\n",
    "    .setSelectClause(\"*\")\\\n",
    "    .setJoinPredicate(\"user_id = user_id4label AND brand_id = brand_id4label\")\\\n",
    "    .linkFrom(t1_join, t2_label);\n",
    "\n",
    "imputer = Imputer()\\\n",
    "    .setStrategy(\"value\")\\\n",
    "    .setFillValue(\"0\")\\\n",
    "    .setSelectedCols([\"label\"]);\n",
    "\n",
    "feature_label = imputer.fit(feature_label).transform(feature_label);\n",
    "\n",
    "print(feature_label.getColNames());\n",
    "\n",
    "featureColNames = feature_label.getColNames()\n",
    "for col_name in [\"user_id\", \"brand_id\",\"user_id4join\", \"brand_id4join\",\n",
    "                 \"user_id4label\", \"brand_id4label\",LABEL_COL_NAME] :\n",
    "    featureColNames.remove(col_name)\n",
    "\n",
    "str = ''\n",
    "for t in featureColNames :\n",
    "    str = str + \"CAST(\" + t + \" AS DOUBLE) AS \" + t + \", \"\n",
    "str = str + LABEL_COL_NAME\n",
    "\n",
    "feature_label\\\n",
    "    .select(str)\\\n",
    "    .link(\n",
    "        AkSinkBatchOp()\\\n",
    "            .setFilePath(DATA_DIR + FEATURE_LABEL_FILE)\\\n",
    "            .setOverwriteSink(True)\n",
    "    );\n",
    "\n",
    "BatchOperator.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_4_1\n",
    "all_data = AkSourceBatchOp().setFilePath(DATA_DIR + FEATURE_LABEL_FILE);\n",
    "\n",
    "all_data\\\n",
    "    .lazyPrintStatistics()\\\n",
    "    .groupBy(\"label\", \"label, COUNT(*) AS cnt\")\\\n",
    "    .print();\n",
    "\n",
    "splitTrainTestIfNotExist(all_data, DATA_DIR + TRAIN_FILE, DATA_DIR + TEST_FILE, 0.8);\n",
    "\n",
    "train_data = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_FILE);\n",
    "test_data = AkSourceBatchOp().setFilePath(DATA_DIR + TEST_FILE);\n",
    "\n",
    "featureColNames = train_data.getColNames()\n",
    "featureColNames.remove(LABEL_COL_NAME)\n",
    "\n",
    "LogisticRegression()\\\n",
    "    .setFeatureCols(featureColNames)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "    .fit(train_data)\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalBinaryClassBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"LogisticRegression\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "\n",
    "if not(os.path.exists(DATA_DIR + TRAIN_SAMPLE_FILE)) :\n",
    "    train_data\\\n",
    "        .link(\n",
    "            StratifiedSampleBatchOp()\\\n",
    "                .setStrataRatios(\"0:0.05,1:1.0\")\\\n",
    "                .setStrataCol(LABEL_COL_NAME)\n",
    "        )\\\n",
    "        .link(\n",
    "            AkSinkBatchOp()\\\n",
    "                .setFilePath(DATA_DIR + TRAIN_SAMPLE_FILE)\n",
    "        );\n",
    "    BatchOperator.execute();\n",
    "\n",
    "\n",
    "train_sample = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_SAMPLE_FILE);\n",
    "\n",
    "LogisticRegression()\\\n",
    "    .setFeatureCols(featureColNames)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "    .fit(train_sample)\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalBinaryClassBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"LogisticRegression with Stratified Sample\")\n",
    "    );\n",
    "BatchOperator.execute();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_4_2\n",
    "train_sample = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_SAMPLE_FILE);\n",
    "test_data = AkSourceBatchOp().setFilePath(DATA_DIR + TEST_FILE);\n",
    "\n",
    "featureColNames = test_data.getColNames()\n",
    "featureColNames.remove(LABEL_COL_NAME)\n",
    "\n",
    "for treeType in ['GINI', 'INFOGAIN', 'INFOGAINRATIO'] :\n",
    "    DecisionTreeClassifier()\\\n",
    "        .setTreeType(treeType)\\\n",
    "        .setFeatureCols(featureColNames)\\\n",
    "        .setLabelCol(LABEL_COL_NAME)\\\n",
    "        .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "        .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "        .fit(train_sample)\\\n",
    "        .transform(test_data)\\\n",
    "        .link(\n",
    "            EvalBinaryClassBatchOp()\\\n",
    "                .setPositiveLabelValueString(\"1\")\\\n",
    "                .setLabelCol(LABEL_COL_NAME)\\\n",
    "                .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "                .lazyPrintMetrics(treeType)\n",
    "        );\n",
    "\n",
    "BatchOperator.execute()\n",
    "\n",
    "RandomForestClassifier()\\\n",
    "    .setNumTrees(20)\\\n",
    "    .setMaxDepth(4)\\\n",
    "    .setMaxBins(512)\\\n",
    "    .setFeatureCols(featureColNames)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "    .fit(train_sample)\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalBinaryClassBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"RandomForest with Stratified Sample\")\n",
    "    );\n",
    "\n",
    "BatchOperator.execute();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_4_3\n",
    "test_data = AkSourceBatchOp().setFilePath(DATA_DIR + TEST_FILE);\n",
    "train_sample = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_SAMPLE_FILE);\n",
    "\n",
    "featureColNames = test_data.getColNames()\n",
    "featureColNames.remove(LABEL_COL_NAME)\n",
    "\n",
    "GbdtClassifier()\\\n",
    "    .setNumTrees(100)\\\n",
    "    .setMaxDepth(5)\\\n",
    "    .setMaxBins(256)\\\n",
    "    .setFeatureCols(featureColNames)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "    .fit(train_sample)\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalBinaryClassBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"GBDT with Stratified Sample\")\n",
    "    );\n",
    "\n",
    "BatchOperator.execute();\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
